package ca.uwindsor.cs.deepweb.utility.reuters;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.LockObtainFailedException;

import ca.uwindsor.cs.deepweb.estimation.DictionariesFilesContentIndexer;
import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;
import ca.uwindsor.cs.deepweb.utility.NekoHTMLHandler;

public class ReutersFilesContentIndexer extends DictionariesFilesContentIndexer {

	public ReutersFilesContentIndexer(String sourcepath, String indexdir,
			String encoding, boolean verbose, boolean create) {
		super(sourcepath, indexdir, encoding, verbose, create,
				new String[] { "xml" });
		// TODO Auto-generated constructor stub
	}

	/**
	 * Start to index the files in the specified directory. One file's content
	 * is an instance of Document in Lucene.
	 */
	public void generateIndex(File directory) {
		try {
			File[] dataFiles = directory.listFiles();
			File reutersxml;
			int size;
			// Add documents to the index
			for (int i = 0; i < dataFiles.length; i++) {
				reutersxml = dataFiles[i];
				if (reutersxml.isFile()) {
					if (isValidExtension(reutersxml.getName())) {

						NekoHTMLHandler handler = new NekoHTMLHandler(encoding);
						try {
							handler.processContent(new FileInputStream(
									dataFiles[i]));
						} catch (Exception e) {
							// e.printStackTrace();
							System.err.println("Error on parsing: "
									+ dataFiles[i].getCanonicalPath());
							continue;
						}

						 String content = handler.getContent();
						 size=content.length();
						// if (content.length() < 1024) {
						// continue;
						// }

						if (isVerbose) {
							System.out.println("Indexing file "
									+ dataFiles[i].getCanonicalPath());
						}

						// System.out.println("Indexing file "
						// + dataFiles[i].getCanonicalPath());
						Document document = new Document();
//						InputStreamReader txtReader = new InputStreamReader(
//								new FileInputStream(reutersxml));

						// make the ID from the path shorter - get the file name
						String file = dataFiles[i].getCanonicalPath();
						int offset1 = file.lastIndexOf(java.io.File.separator);
						int offset2 = file.lastIndexOf(".");
						String word = new String(file.substring(offset1 + 1,
								offset2));
						//
						// String word = String.valueOf(id);
						// System.out.println(id);

						Field FieldPath = new Field(
								FilesContentIndexer.FIELD_ID, word,
								Field.Store.YES, Field.Index.UN_TOKENIZED);

						Field FieldTitle = new Field(
								FilesContentIndexer.FIELD_TITLE, handler.getTitle(),
								Field.Store.YES, Field.Index.UN_TOKENIZED);
						
						Field FieldSize = new Field(
								FilesContentIndexer.FIELD_SIZE, Integer.toString(size),
								Field.Store.YES, Field.Index.UN_TOKENIZED);
						
						Field FieldBody = new Field(
								FilesContentIndexer.FIELD_CONTENT, content,
								Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES);

						document.add(FieldPath);
						document.add(FieldTitle);
						document.add(FieldSize);
						document.add(FieldBody);
						indexWriter.addDocument(document);

						globalID++;
					}
				} else if (reutersxml.isDirectory()) {
					generateIndex(reutersxml);
				}
			}
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		}

	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		
//		args = new String[7];
//		args[0] = "--SourceDir=D:\\Research\\estimation\\datasource\\Reuters_regrouped";
//		args[1] = "--IndexDir=F:\\Reuters_grouped";
//		args[2] = "--encoding=UTF-8";
//		args[3] = "-verbose";
//		args[4]= "-create";
//		args[5]= "-resursive";
//		args[6]= "--extension=xml";	

		String SourceDir = new String();
		String IndexDir = new String();
		String encoding = new String("UTF-8");
		boolean isVerbose = false;
		boolean isCreate = true;
		boolean isRecursive = false;

		String ext[] = new String[] { "xml" };
		
		if (args.length < 2) {
			System.out
					.println("usage: java -jar ReutersFilesContentIndexer --SourceDir=sourcefiledirectory --IndexDir=indexfiledirectory --extension=txt,html,... [--encoding=encoding](default=UTF-8) [-verbose] [-create] !");
			System.exit(0);
		} else {
			for (int i = 0; i < args.length; i++) {
				if (args[i].startsWith("--SourceDir=")) {
					SourceDir = args[i].substring(12);
				} else if (args[i].startsWith("--IndexDir=")) {
					IndexDir = args[i].substring(11);
				} else if (args[i].startsWith("--encoding=")) {
					encoding = args[i].substring(11);
				} else if (args[i].equals("-create")) {
					isVerbose = true;
				} else if (args[i].equals("-verbose")) {
					isCreate = true;
				} else if (args[i].startsWith("--extension=")) {
					String tempexts = args[i].substring(12);
					ext = tempexts.split(",");
				} else if (args[i].endsWith("-resursive")) {
					isRecursive = true;
				}
			}
		}

		if (SourceDir.equals("") || IndexDir.equals("")) {
			System.out
					.println("usage: java -jar ReutersFilesContentIndexer SourceDir=sourcefiledirectory IndexDir=indexfiledirectory [encoding=encoding](default=UTF-8) [-verbose]!");
			System.exit(0);
		} else {
			ReutersFilesContentIndexer indexer ;
//			if (isRecursive) {
//				indexer = new HTMLFilesContentIndexer(SourceDir, IndexDir, encoding,
//						isVerbose, isCreate, ext);
//			} else {
//				indexer = new HTMLFilesContentIndexer (SourceDir, IndexDir, encoding,
//						isVerbose, isCreate, ext);
//			}
			indexer = new ReutersFilesContentIndexer(SourceDir, IndexDir, encoding,
					isVerbose, isCreate);
			indexer.generateIndex();
		}
	}

}
